package com.simple.database.pageprocess;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.concurrent.atomic.AtomicLong;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.model.AfterExtractor;
import us.codecraft.webmagic.model.OOSpider;
import us.codecraft.webmagic.model.annotation.ExtractBy;
import us.codecraft.webmagic.model.annotation.HelpUrl;
import us.codecraft.webmagic.model.annotation.TargetUrl;
import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.scheduler.component.HashSetDuplicateRemover;

import com.alibaba.fastjson.JSONObject;
import com.simple.database.mapper.BlogMapper;
import com.simple.database.pojo.Blog;
import com.simple.database.service.RedisService;
import com.simple.database.utils.RedisUtil;

@HelpUrl("http://blog.csdn.net/\\w+")
@TargetUrl("http://blog.csdn.net/\\w+/\\w+/\\w+/\\w+")
public class AnnotationBlog2PageProcess implements AfterExtractor {

	Logger logger =LoggerFactory.getLogger(this.getClass());
	
    @ExtractBy("//div[@id='blog_title']/h2/a/text()")
    private String author;

    @ExtractBy(value="//div[@class='article_title']/h1/span/a/text()",notNull=true)
    private String title;

    @ExtractBy("//div[@class='article_title']/h1/span/a/@href")
    private String url;

    @ExtractBy("//span[@class='link_view']/text()")
    private String views;

    @ExtractBy("//span[@class='link_comments']/text()")
    private String comments;

    @ExtractBy("//span[@class='link_postdate']/text()")
    private String writeTime;
	
	
	// 计数器
	private static final AtomicLong count = new AtomicLong(0);

	//加载spring配置文件
	ApplicationContext ac = new ClassPathXmlApplicationContext("spring/applicationContext*.xml");
	
//	private BlogMapper blogMapper;
	
	private RedisService redisService;

	    public static void main(String[] args) {
	        OOSpider.create(Site.me().setRetryTimes(3).setSleepTime(3000).setCharset("utf-8"),AnnotationBlog2PageProcess.class)
	        .addUrl("http://blog.csdn.net/")
	        .setScheduler(new QueueScheduler().setDuplicateRemover(new HashSetDuplicateRemover()))
	        .setScheduler(new FileCacheQueueScheduler("E:\\temp\\webmagic"))
	        .thread(4)
	        .run();
	    }

		@Override
		public void afterProcess(Page page) {
//			blogMapper = ac.getBean(BlogMapper.class);
			redisService = ac.getBean(RedisService.class);
			System.out.println("=================>title:"+title);
			Blog blog = new Blog();
	        	count.incrementAndGet();
	        	System.out.println("抓取到的第："+count.get()+"条有效数据");
				blog.setTitle(title.trim());
				blog.setUrl(url.trim());
				blog.setAuthor(author.trim());
				views = views==null?"":views.trim().replace("(", "").replace(")", "").replace("人阅读", "");//String.valueOf(page.getResultItems().get("views")==null?"":page.getResultItems().get("views"));
				blog.setViews(Integer.valueOf(views));
				blog.setComments(Integer.valueOf(comments.trim().replace("(", "").replace(")", "")));
				Date writeDate = null;
				try {
					writeDate = new SimpleDateFormat("yyyy-MM-dd HH:mm").parse(writeTime.trim());
				} catch (ParseException e1) {
					e1.printStackTrace();
					logger.error("时间转换出错：{}",e1.getMessage());
				}
				blog.setWriteTime(writeDate);
				blog.setInsertTime(new Date());
				blog.setSid(2);
				blog.setId(count.intValue());
				RedisUtil.set(blog.getUrl(), JSONObject.toJSONString(blog));
				//以后用nosql做
//				blogMapper.insert(blog);
//				logger.warn("====>成功插入数据:{}",blog.getUrl());;
		}

}
